package com.coderfans.utils;

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Created by ${fanchunshuai} on 2017-7-19.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2017 58到家-版权所有
 */
public class ReadCsdnUserLog2 {
    public static void main(String[] args) throws IOException {
        File file = new File("E:\\spiderFile\\csdnUserLink2.txt");
        File file2 = new File("E:\\spiderFile\\csdnUserLink3.txt");
        Set<String> urlSet = new HashSet<>();
        List<String> list = FileUtils.readLines(file,"UTF-8");
        String [] urlArray1 = new String[list.size()];

        for (int i = 0 ;i < list.size();i++ ){
            urlArray1[i] = list.get(i).replace("\n","");
        }
        Collections.addAll(urlSet,urlArray1);

        List<String> list2 = FileUtils.readLines(file2,"UTF-8");
        String [] urlArray2 = new String[list2.size()];

        for (int i = 0 ;i < list2.size();i++ ){
            urlArray2[i] = list2.get(i).replace("\n","");
        }

        Collections.addAll(urlSet,urlArray2);

        System.out.println(urlSet.size()+"---------------");
        File userFile = new File("E:\\spiderFile\\csdnUserLinkAll.txt");
        Iterator<String> it = urlSet.iterator();
        int i = 0;
        StringBuffer buffer = new StringBuffer();
        while (it.hasNext()){
            i++;
            buffer.append(it.next()+",");
            if(i % 10 == 0){
                buffer.append("\n");
            }
        }
        FileUtils.writeStringToFile(userFile,buffer.toString());
    }
}
